{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QFdG4eYf3h0L"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/rag/haystack_mongodb_cooking_advisor_pipeline.ipynb)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rrobdhRcNb5I"
      },
      "source": [
        "# Haystack and MongoDB Atlas RAG notebook\n",
        "\n",
        "Install dependencies:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "76dK0ehtNY2L",
        "outputId": "c9510cec-db48-43f6-e81f-6df044393351"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting haystack-ai\n",
            "  Using cached haystack_ai-2.2.1-py3-none-any.whl (345 kB)\n",
            "Collecting mongodb-atlas-haystack\n",
            "  Using cached mongodb_atlas_haystack-0.3.0-py3-none-any.whl (13 kB)\n",
            "Collecting tiktoken\n",
            "  Using cached tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "Collecting datasets\n",
            "  Using cached datasets-2.19.2-py3-none-any.whl (542 kB)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (3.1.4)\n",
            "Collecting lazy-imports (from haystack-ai)\n",
            "  Downloading lazy_imports-0.3.1-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (10.1.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (3.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (1.25.2)\n",
            "Collecting openai>=1.1.0 (from haystack-ai)\n",
            "  Downloading openai-1.34.0-py3-none-any.whl (325 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.5/325.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (2.0.3)\n",
            "Collecting posthog (from haystack-ai)\n",
            "  Downloading posthog-3.5.0-py2.py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (2.8.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (6.0.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (2.31.0)\n",
            "Requirement already satisfied: tenacity in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (8.3.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions>=4.7 in /usr/local/lib/python3.10/dist-packages (from haystack-ai) (4.12.2)\n",
            "Collecting pymongo[srv] (from mongodb-atlas-haystack)\n",
            "  Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2024.5.15)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.14.0)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting requests (from haystack-ai)\n",
            "  Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.5)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.2 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.23.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->haystack-ai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->haystack-ai) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai>=1.1.0->haystack-ai)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->haystack-ai) (2.7.3)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->haystack-ai) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->haystack-ai) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->haystack-ai) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->haystack-ai) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->haystack-ai) (2024.6.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->haystack-ai) (2.1.5)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->haystack-ai) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->haystack-ai) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil->haystack-ai) (1.16.0)\n",
            "Collecting monotonic>=1.5 (from posthog->haystack-ai)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog->haystack-ai)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo[srv]->mongodb-atlas-haystack)\n",
            "  Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.1.0->haystack-ai) (1.2.1)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.1.0->haystack-ai)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.1.0->haystack-ai)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.1.0->haystack-ai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.1.0->haystack-ai) (2.18.4)\n",
            "Installing collected packages: monotonic, xxhash, requests, lazy-imports, h11, dnspython, dill, backoff, tiktoken, pymongo, posthog, multiprocess, httpcore, httpx, openai, datasets, haystack-ai, mongodb-atlas-haystack\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.31.0\n",
            "    Uninstalling requests-2.31.0:\n",
            "      Successfully uninstalled requests-2.31.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.32.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed backoff-2.2.1 datasets-2.19.2 dill-0.3.8 dnspython-2.6.1 h11-0.14.0 haystack-ai-2.2.1 httpcore-1.0.5 httpx-0.27.0 lazy-imports-0.3.1 mongodb-atlas-haystack-0.3.0 monotonic-1.6 multiprocess-0.70.16 openai-1.34.0 posthog-3.5.0 pymongo-4.7.3 requests-2.32.3 tiktoken-0.7.0 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "pip install haystack-ai mongodb-atlas-haystack tiktoken datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aeg_wcIiPYnY"
      },
      "source": [
        "\n",
        "## Setup MongoDB Atlas connection and Open AI\n",
        "\n",
        "\n",
        "* Set the MongoDB connection string. Follow the steps [here](https://www.mongodb.com/docs/manual/reference/connection-string/) to get the connection string from the Atlas UI.\n",
        "\n",
        "* Set the OpenAI API key. Steps to obtain an API key as [here](https://help.openai.com/en/articles/4936850-where-do-i-find-my-openai-api-key)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "MZokdDxIPb9p"
      },
      "outputs": [],
      "source": [
        "import getpass\n",
        "import os\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "57gYJTBVPfBX",
        "outputId": "3bfa3a55-e7e1-44db-9c20-2dfdaa8e76f2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your MongoDB connection string:··········\n"
          ]
        }
      ],
      "source": [
        "conn_str = getpass.getpass(\"Enter your MongoDB connection string:\")\n",
        "conn_str = conn_str.strip()\n",
        "conn_str = (\n",
        "    re.sub(\n",
        "        r\"appName=[^&\\s]*\",\n",
        "        \"appName=devrel.content.python\",\n",
        "        conn_str,\n",
        "        flags=re.IGNORECASE,\n",
        "    )\n",
        "    if re.search(r\"appName=[^&\\s]*\", conn_str, re.IGNORECASE)\n",
        "    else conn_str + (\"&\" if \"?\" in conn_str else \"?\") + \"appName=devrel.content.python\"\n",
        ")\n",
        "os.environ[\"MONGO_CONNECTION_STRING\"] = conn_str"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J8Gd-SMuRSH-",
        "outputId": "148dd391-c35d-482e-e075-8094aa556eae"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your Open AI Key:··········\n"
          ]
        }
      ],
      "source": [
        "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your Open AI Key:\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HnG43-yPRIl3"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fv1pPHqXQFa-"
      },
      "source": [
        "## Create vector search index on collection\n",
        "\n",
        "Follow this [tutorial](https://www.mongodb.com/docs/atlas/atlas-vector-search/create-index/) to create a vector index on database: `ai_shop` collection `test_collection`.\n",
        "\n",
        "Verify that the index name is `vector_index` and the syntax specify:\n",
        "```\n",
        "{\n",
        "  \"fields\": [\n",
        "    {\n",
        "      \"type\": \"vector\",\n",
        "      \"path\": \"embedding\",\n",
        "      \"numDimensions\": 1536,\n",
        "      \"similarity\": \"cosine\"\n",
        "    }\n",
        "  ]\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cOMyplbvOMDk"
      },
      "source": [
        "### Setup vector store to load documents:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "-y9waymAOOgs"
      },
      "outputs": [],
      "source": [
        "from bson import json_util\n",
        "from haystack import Document, Pipeline\n",
        "from haystack.components.builders.prompt_builder import PromptBuilder\n",
        "from haystack.components.embedders import OpenAIDocumentEmbedder, OpenAITextEmbedder\n",
        "from haystack.components.generators import OpenAIGenerator\n",
        "from haystack.components.writers import DocumentWriter\n",
        "from haystack.document_stores.types import DuplicatePolicy\n",
        "from haystack_integrations.components.retrievers.mongodb_atlas import (\n",
        "    MongoDBAtlasEmbeddingRetriever,\n",
        ")\n",
        "from haystack_integrations.document_stores.mongodb_atlas import (\n",
        "    MongoDBAtlasDocumentStore,\n",
        ")\n",
        "\n",
        "# Create some example documents\n",
        "# documents = [\n",
        "#     Document(content=\"My name is Jean and I live in Paris.\"),\n",
        "#     Document(content=\"My name is Mark and I live in Berlin.\"),\n",
        "#     Document(content=\"My name is Giorgio and I live in Rome.\"),\n",
        "# ]\n",
        "\n",
        "# dataset = load_dataset(\"MongoDB/product-catalog\")\n",
        "\n",
        "dataset = {\n",
        "    \"train\": [\n",
        "        {\n",
        "            \"title\": \"Spinach Lasagna Sheets\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"Infused with spinach, these sheets add a pop of color and extra nutrients.\",\n",
        "            \"category\": \"Pasta\",\n",
        "            \"emoji\": \"📗\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Gluten-Free Lasagna Sheets\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"Perfect for those with gluten intolerance, made with a blend of rice and corn flour.\",\n",
        "            \"category\": \"Pasta\",\n",
        "            \"emoji\": \"🍚🌽\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Mascarpone\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"Creamy and rich, this cheese adds a luxurious touch to lasagna.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Queso Fresco\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"A mild, crumbly cheese that can be a suitable replacement for ricotta.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegetarian Lentil Sauce\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A meatless option made with cooked lentils that mimics the texture of ground meat.\",\n",
        "            \"category\": \"Vegetarian\",\n",
        "            \"emoji\": \"🍲\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Turkey Bolognese\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"A leaner alternative to beef, turkey provides a lighter but flavorful taste.\",\n",
        "            \"category\": \"Poultry\",\n",
        "            \"emoji\": \"🦃\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Mushroom and Walnut Sauce\",\n",
        "            \"price\": \"$5.50\",\n",
        "            \"description\": \"Combining chopped mushrooms and walnuts for a hearty vegetarian filling.\",\n",
        "            \"category\": \"Vegetarian\",\n",
        "            \"emoji\": \"🍄🥜\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Chicken Bolognese\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"Ground chicken offers a different twist on the classic meat sauce.\",\n",
        "            \"category\": \"Poultry\",\n",
        "            \"emoji\": \"🐔\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegan Soy Meat Sauce\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"Made from soy protein, this vegan meat sauce replicates the texture and flavor of traditional meat.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🌱\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Tomato Basil Sauce\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"A tangy alternative to béchamel, made with fresh tomatoes and basil.\",\n",
        "            \"category\": \"Vegetarian\",\n",
        "            \"emoji\": \"🍅\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Pesto Cream Sauce\",\n",
        "            \"price\": \"$5.50\",\n",
        "            \"description\": \"A fusion of creamy béchamel and rich basil pesto for a unique flavor.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🍝\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Alfredo Sauce\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"A rich and creamy white sauce made with parmesan and butter.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧈\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Coconut Milk Béchamel\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A dairy-free version of the classic béchamel made with coconut milk.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🥥\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegan Cashew Cream Sauce\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"A rich and creamy sauce made from blended cashews as a dairy-free alternative.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🥜\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Kale\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Another leafy green option, kale offers a chewy texture and rich nutrients.\",\n",
        "            \"category\": \"Leafy Greens\",\n",
        "            \"emoji\": \"🥬\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Bell Peppers\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Sliced bell peppers in various colors add sweetness and crunch.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🫑\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Artichoke Hearts\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"Tender and flavorful, artichoke hearts bring a Mediterranean twist to the dish.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🍽️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Spinach\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Fresh or frozen spinach adds a pop of color and nutrients.\",\n",
        "            \"category\": \"Leafy Greens\",\n",
        "            \"emoji\": \"🥬\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Broccoli\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Small broccoli florets provide texture and a distinct flavor.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🥦\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Whole Wheat Lasagna Sheets\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"Made from whole wheat grains, these sheets are healthier and provide a nutty flavor.\",\n",
        "            \"category\": \"Pasta\",\n",
        "            \"emoji\": \"🌾\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Zucchini Slices\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Thinly sliced zucchini can replace traditional pasta for a low-carb version.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🥒\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Eggplant Slices\",\n",
        "            \"price\": \"$2.75\",\n",
        "            \"description\": \"Thin slices of eggplant provide a meaty texture, ideal for vegetarian lasagna.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🍆\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Ground Turkey\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"A leaner alternative to beef, turkey provides a lighter but flavorful taste.\",\n",
        "            \"category\": \"Meat\",\n",
        "            \"emoji\": \"🦃\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegetarian Lentil Mince\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"A meatless option made with cooked lentils that mimics the texture of ground meat.\",\n",
        "            \"category\": \"Vegetarian\",\n",
        "            \"emoji\": \"🍲\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Mushroom and Walnut Mince\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"Combining chopped mushrooms and walnuts for a hearty vegetarian filling.\",\n",
        "            \"category\": \"Vegetarian\",\n",
        "            \"emoji\": \"🍄🥜\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Ground Chicken\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"Ground chicken offers a different twist on the classic meat sauce.\",\n",
        "            \"category\": \"Poultry\",\n",
        "            \"emoji\": \"🐔\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegan Soy Meat Crumbles\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"Made from soy protein, these crumbles replicate the texture and flavor of traditional meat.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🥩\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Pesto Sauce\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A green, aromatic sauce made from basil, pine nuts, and garlic.\",\n",
        "            \"category\": \"Canned Goods\",\n",
        "            \"emoji\": \"🌿\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Marinara Sauce\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"A classic Italian tomato sauce with garlic, onions, and herbs.\",\n",
        "            \"category\": \"Canned Goods\",\n",
        "            \"emoji\": \"🍅\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Bolognese Sauce\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"A meat-based sauce simmered with tomatoes, onions, celery, and carrots.\",\n",
        "            \"category\": \"Canned Goods\",\n",
        "            \"emoji\": \"🍖🍅🧅🥕\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Arrabbiata Sauce\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A spicy tomato sauce made with red chili peppers.\",\n",
        "            \"category\": \"Canned Goods\",\n",
        "            \"emoji\": \"🌶️🍅\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Provolone Cheese\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"Semi-hard cheese with a smooth texture, it melts beautifully in dishes.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Cheddar Cheese\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"A popular cheese with a sharp and tangy flavor profile.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Gouda Cheese\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"A Dutch cheese known for its rich and creamy texture.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Fontina Cheese\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A semi-soft cheese with a strong flavor, great for melting.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Vegan Mozzarella\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"Dairy-free alternative made from nuts or soy, melts similarly to regular mozzarella.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Cottage Cheese\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"A lighter alternative to ricotta, with small curds that provide a similar texture.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Goat Cheese\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A tangy and creamy cheese that can provide a unique flavor to lasagna.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Mascarpone Cheese\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"An Italian cream cheese with a rich and creamy texture.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Tofu Ricotta\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"A vegan alternative made from crumbled tofu seasoned with herbs.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🌱\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Feta Cheese\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"A crumbly cheese with a salty profile, it can bring a Mediterranean twist to the dish.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Parmesan cheese\",\n",
        "            \"price\": \"$4.00\",\n",
        "            \"description\": \"A hard, granular cheese originating from Italy, known for its rich umami flavor.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Pecorino Romano\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"A salty, hard cheese made from sheep's milk, perfect for grating over dishes.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Asiago Cheese\",\n",
        "            \"price\": \"$4.50\",\n",
        "            \"description\": \"Semi-hard cheese with a nutty flavor, great for shaving or grating.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Grana Padano\",\n",
        "            \"price\": \"$5.50\",\n",
        "            \"description\": \"A grainy, hard cheese that's similar to Parmesan but milder in flavor.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Manchego Cheese\",\n",
        "            \"price\": \"$6.00\",\n",
        "            \"description\": \"A Spanish hard cheese with a rich and nutty flavor.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🧀\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Eggs\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Rich in protein and versatile, eggs are used in a variety of culinary applications.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🥚\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Tofu\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Blended silken tofu can act as a binder in various dishes.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🍲\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Flaxseed Meal\",\n",
        "            \"price\": \"$1.50\",\n",
        "            \"description\": \"Mix with water to create a gel-like consistency that can replace eggs.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🥚\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Chia Seeds\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Mix with water to form a gel that can be used as an egg substitute.\",\n",
        "            \"category\": \"Vegan\",\n",
        "            \"emoji\": \"🥚\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Apple Sauce\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"A sweet alternative that can replace eggs in certain recipes.\",\n",
        "            \"category\": \"Baking\",\n",
        "            \"emoji\": \"🥚\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Onion\",\n",
        "            \"price\": \"$1.00\",\n",
        "            \"description\": \"A kitchen staple, onions provide depth and flavor to a myriad of dishes.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🧅\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Shallots\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Milder and sweeter than regular onions, they add a delicate flavor.\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🧅\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Green Onions\",\n",
        "            \"price\": \"$1.50\",\n",
        "            \"description\": \"Milder in flavor, green onions or scallions are great for garnishing.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🌱\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Red Onion\",\n",
        "            \"price\": \"$1.20\",\n",
        "            \"description\": \"Sweeter and more vibrant in color, red onions add a pop to dishes.\",\n",
        "            \"category\": \"Vegetables\",\n",
        "            \"emoji\": \"🔴\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Leeks\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"With a light onion flavor, leeks are great when sautéed or used in soups.\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🍲\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Garlic\",\n",
        "            \"price\": \"$0.50\",\n",
        "            \"description\": \"Aromatic and flavorful, garlic is a foundational ingredient in many cuisines.\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🧄\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Garlic Powder\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"A convenient dried version of garlic that provides a milder flavor.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧄\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Garlic Flakes\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Dried garlic flakes can be rehydrated or used as they are for a burst of garlic flavor.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧄\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Garlic Paste\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"A smooth blend of garlic, perfect for adding to sauces or marinades.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🧄\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Olive Oil\",\n",
        "            \"price\": \"$6.00\",\n",
        "            \"description\": \"A staple in Mediterranean cuisine, olive oil is known for its heart-healthy properties.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🍽️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Canola Oil\",\n",
        "            \"price\": \"$3.50\",\n",
        "            \"description\": \"A neutral-tasting oil suitable for various cooking methods.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🍳\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Coconut Oil\",\n",
        "            \"price\": \"$5.00\",\n",
        "            \"description\": \"A fragrant oil ideal for sautéing and baking.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🍳\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Avocado Oil\",\n",
        "            \"price\": \"$7.00\",\n",
        "            \"description\": \"Known for its high smoke point, it's great for high-heat cooking.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🍳\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Grapeseed Oil\",\n",
        "            \"price\": \"$6.50\",\n",
        "            \"description\": \"A light, neutral oil that's good for dressings and sautéing.\",\n",
        "            \"category\": \"Condiments\",\n",
        "            \"emoji\": \"🥗\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Salt\",\n",
        "            \"price\": \"$1.00\",\n",
        "            \"description\": \"An essential seasoning that enhances the flavor of dishes.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧂\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Himalayan Pink Salt\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"A natural and unrefined salt with a slightly earthy flavor.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧂\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Sea Salt\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Derived from evaporated seawater, it provides a briny touch.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🌊\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Kosher Salt\",\n",
        "            \"price\": \"$1.50\",\n",
        "            \"description\": \"A coarse salt without additives, commonly used in cooking.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧂\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Black Salt (Kala Namak)\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"A sulfurous salt often used in South Asian cuisine, especially vegan dishes to mimic an eggy flavor.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🧂\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Black Pepper\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"A versatile spice known for its sharp and mildly spicy flavor.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🌶️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"White Pepper\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"Milder than black pepper, it's often used in light-colored dishes.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🌶️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Cayenne Pepper\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"A spicy chili pepper, ground into powder. Adds heat to dishes.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🌶️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Crushed Red Pepper Flakes\",\n",
        "            \"price\": \"$1.50\",\n",
        "            \"description\": \"Adds a spicy kick to dishes, commonly used as a pizza topping.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🌶️\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Sichuan (or Szechuan) Peppercorns\",\n",
        "            \"price\": \"$3.00\",\n",
        "            \"description\": \"Known for their unique tingling sensation, they're used in Chinese cuisine.\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🥡\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Banana\",\n",
        "            \"price\": \"$0.60\",\n",
        "            \"description\": \"A sweet and portable fruit, packed with essential vitamins.\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🍌\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Milk\",\n",
        "            \"price\": \"$2.50\",\n",
        "            \"description\": \"A calcium-rich dairy product, perfect for drinking or cooking.\",\n",
        "            \"category\": \"Dairy\",\n",
        "            \"emoji\": \"🥛\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Bread\",\n",
        "            \"price\": \"$2.00\",\n",
        "            \"description\": \"Freshly baked, perfect for sandwiches or toast.\",\n",
        "            \"category\": \"Bakery\",\n",
        "            \"emoji\": \"🍞\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Apple\",\n",
        "            \"price\": \"$1.00\",\n",
        "            \"description\": \"Crisp and juicy, great for snacking or baking.\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🍏\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Orange\",\n",
        "            \"price\": \"3.99$\",\n",
        "            \"description\": \"Great as a juice and vitamin\",\n",
        "            \"category\": \"Produce\",\n",
        "            \"emoji\": \"🍊\",\n",
        "        },\n",
        "        {\n",
        "            \"title\": \"Sugar\",\n",
        "            \"price\": \"1.00\",\n",
        "            \"description\": \"very sweet substance\",\n",
        "            \"category\": \"Spices\",\n",
        "            \"emoji\": \"🍰\",\n",
        "        },\n",
        "    ]\n",
        "}\n",
        "\n",
        "insert_data = []\n",
        "\n",
        "for product in dataset[\"train\"]:\n",
        "    doc_product = json_util.loads(json_util.dumps(product))\n",
        "    haystack_doc = Document(content=doc_product[\"title\"], meta=doc_product)\n",
        "    insert_data.append(haystack_doc)\n",
        "\n",
        "\n",
        "document_store = MongoDBAtlasDocumentStore(\n",
        "    database_name=\"ai_shop\",\n",
        "    collection_name=\"test_collection\",\n",
        "    vector_search_index=\"vector_index\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3MMitwR3P0uj"
      },
      "source": [
        "Build the writer pipeline to load documnets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dYEo2ZkMQptv",
        "outputId": "f07e014d-346b-4897-e8a2-2f1fbac9a97e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Calculating embeddings: 100%|██████████| 3/3 [00:01<00:00,  2.85it/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'doc_embedder': {'meta': {'model': 'text-embedding-ada-002',\n",
              "   'usage': {'prompt_tokens': 310, 'total_tokens': 310}}},\n",
              " 'doc_writer': {'documents_written': 81}}"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Setting up a document writer to handle the insertion of documents into the MongoDB collection.\n",
        "doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP)\n",
        "\n",
        "# Initializing a document embedder to convert text content into vectorized form.\n",
        "doc_embedder = OpenAIDocumentEmbedder()\n",
        "\n",
        "# Creating a pipeline for indexing documents. The pipeline includes embedding and writing documents.\n",
        "indexing_pipe = Pipeline()\n",
        "indexing_pipe.add_component(instance=doc_embedder, name=\"doc_embedder\")\n",
        "indexing_pipe.add_component(instance=doc_writer, name=\"doc_writer\")\n",
        "\n",
        "# Connecting the components of the pipeline for document flow.\n",
        "indexing_pipe.connect(\"doc_embedder.documents\", \"doc_writer.documents\")\n",
        "\n",
        "# Running the pipeline with the list of documents to index them in MongoDB.\n",
        "indexing_pipe.run({\"doc_embedder\": {\"documents\": insert_data}})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TA_ELb_URY0D"
      },
      "source": [
        "## Build a RAG Pipeline\n",
        "\n",
        "Lets create a pipeline that will Retrieve Augment and Generate a response for user questions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RaNFayobRkU9",
        "outputId": "9d14fa6c-c85f-442a-8736-7672502edd17"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<haystack.core.pipeline.pipeline.Pipeline object at 0x7e7804231bd0>\n",
              "🚅 Components\n",
              "  - text_embedder: OpenAITextEmbedder\n",
              "  - retriever: MongoDBAtlasEmbeddingRetriever\n",
              "  - prompt_builder: PromptBuilder\n",
              "  - llm: OpenAIGenerator\n",
              "🛤️ Connections\n",
              "  - text_embedder.embedding -> retriever.query_embedding (List[float])\n",
              "  - retriever.documents -> prompt_builder.documents (List[Document])\n",
              "  - prompt_builder.prompt -> llm.prompt (str)"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Template for generating prompts for a movie recommendation engine.\n",
        "prompt_template = \"\"\"\n",
        "    You are a chef assistant allowed to use the following context documents and only those.\\nDocuments:\n",
        "    {% for doc in documents %}\n",
        "        {{ doc.content }}\n",
        "    {% endfor %}\n",
        "\n",
        "    \\Query: {{query}}\n",
        "\n",
        "    \\nAnswer:\n",
        "\"\"\"\n",
        "\n",
        "# Setting up a retrieval-augmented generation (RAG) pipeline for generating responses.\n",
        "rag_pipeline = Pipeline()\n",
        "rag_pipeline.add_component(\"text_embedder\", OpenAITextEmbedder())\n",
        "\n",
        "# Adding a component for retrieving related documents from MongoDB based on the query embedding.\n",
        "rag_pipeline.add_component(\n",
        "    instance=MongoDBAtlasEmbeddingRetriever(document_store=document_store, top_k=50),\n",
        "    name=\"retriever\",\n",
        ")\n",
        "\n",
        "# Building prompts based on retrieved documents to be used for generating responses.\n",
        "rag_pipeline.add_component(\n",
        "    instance=PromptBuilder(template=prompt_template), name=\"prompt_builder\"\n",
        ")\n",
        "\n",
        "# Adding a language model generator to produce the final text output.\n",
        "rag_pipeline.add_component(instance=OpenAIGenerator(model=\"gpt-4o\"), name=\"llm\")\n",
        "\n",
        "# Connecting the components of the RAG pipeline to ensure proper data flow.\n",
        "rag_pipeline.connect(\"text_embedder.embedding\", \"retriever.query_embedding\")\n",
        "rag_pipeline.connect(\"retriever\", \"prompt_builder.documents\")\n",
        "rag_pipeline.connect(\"prompt_builder\", \"llm\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7mEU3ydRz8k"
      },
      "source": [
        "Lets test the pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GHV5rRhaR3SX",
        "outputId": "793751aa-0c1b-4591-bb8e-fdcd276e11df"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "To cook a lasagne, you can follow this classic recipe:\n",
            "\n",
            "### Ingredients:\n",
            "#### For the meat sauce:\n",
            "- 2 tablespoons olive oil\n",
            "- 1 onion, finely chopped\n",
            "- 2 cloves garlic, minced\n",
            "- 500g ground beef\n",
            "- 800g canned tomatoes, crushed\n",
            "- 2 tablespoons tomato paste\n",
            "- 1 teaspoon dried basil\n",
            "- 1 teaspoon dried oregano\n",
            "- Salt and pepper to taste\n",
            "\n",
            "#### For the béchamel sauce:\n",
            "- 4 tablespoons butter\n",
            "- 4 tablespoons all-purpose flour\n",
            "- 500ml milk\n",
            "- A pinch of nutmeg\n",
            "- Salt and pepper to taste\n",
            "\n",
            "#### For assembly:\n",
            "- 250g lasagne sheets\n",
            "- 200g mozzarella cheese, shredded\n",
            "- 1 cup grated Parmesan cheese\n",
            "- Fresh basil leaves for garnish (optional)\n",
            "\n",
            "### Instructions:\n",
            "1. **Preheat the oven** to 375°F (190°C).\n",
            "\n",
            "2. **Prepare the meat sauce:**\n",
            "   - Heat the olive oil in a large skillet over medium heat.\n",
            "   - Add the chopped onion and cook until soft and translucent, about 5 minutes.\n",
            "   - Stir in the minced garlic and cook for another minute.\n",
            "   - Add the ground beef and cook until browned, breaking it up with a spoon as it cooks.\n",
            "   - Stir in the crushed tomatoes, tomato paste, dried basil, and dried oregano.\n",
            "   - Season with salt and pepper, then reduce the heat to low.\n",
            "   - Let the sauce simmer for 30 minutes, stirring occasionally.\n",
            "\n",
            "3. **Prepare the béchamel sauce:**\n",
            "   - In a medium saucepan, melt the butter over medium heat.\n",
            "   - Add the flour and whisk continuously for about 2 minutes to create a roux.\n",
            "   - Gradually add the milk while whisking to prevent lumps from forming.\n",
            "   - Cook the mixture, whisking constantly, until it thickens, about 5-7 minutes.\n",
            "   - Season with a pinch of nutmeg, salt, and pepper.\n",
            "\n",
            "4. **Assemble the lasagne:**\n",
            "   - Spread a thin layer of the meat sauce on the bottom of a 9x13 inch baking dish.\n",
            "   - Place a layer of lasagne sheets over the sauce.\n",
            "   - Spread another layer of meat sauce over the lasagne sheets, followed by a layer of béchamel sauce.\n",
            "   - Sprinkle some shredded mozzarella cheese over the béchamel sauce.\n",
            "   - Repeat the layers until all the ingredients are used, finishing with a layer of béchamel sauce and a generous topping of mozzarella and Parmesan cheese.\n",
            "\n",
            "5. **Bake the lasagne:**\n",
            "   - Cover the baking dish with aluminum foil.\n",
            "   - Bake in the preheated oven for 30 minutes.\n",
            "   - Remove the foil and bake for an additional 15 minutes, or until the top is golden brown and bubbling.\n",
            "\n",
            "6. **Rest and serve:**\n",
            "   - Remove the lasagne from the oven and let it rest for 10-15 minutes before slicing.\n",
            "   - Garnish with fresh basil leaves if desired, and serve.\n",
            "\n",
            "Enjoy your delicious homemade lasagne!\n"
          ]
        }
      ],
      "source": [
        "query = \"How can I cook a lasagne?\"\n",
        "result = rag_pipeline.run(\n",
        "    {\n",
        "        \"text_embedder\": {\"text\": query},\n",
        "        \"prompt_builder\": {\"query\": query},\n",
        "    }\n",
        ")\n",
        "print(result[\"llm\"][\"replies\"][0])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "state": {}
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
